import scrapy
import os
from scrapy import cmdline
from ff import items
import json
import logging
from icecream import ic


class XlSpider(scrapy.Spider):  # 选择的爬虫引擎是scrapy.Spider
    name = 'xl'
    allowed_domains = ['huya.com']  # 如果网页地址不在此字符串内,不会爬取,若要爬取,注释此行

    start_urls = [f'https://www.huya.com/cache.php?m=LiveList&do=getLiveListByPage&gameId=1663&tagAll=0&page={i}' for i in range(1, 6)]

    # def start_requests(self):
    #     for i in range(1,6):
    #         url = f'https://www.huya.com/cache.php?m=LiveList&do=getLiveListByPage&gameId=1663&tagAll=0&page={i}'
    #         scrapy.Request(url,callback = self.parse)

    def parse(self, response):
        # item = items.FfItem()
        # # print(response.text)
        # item['content'] = response.text
        # ic(item)
        # return item
        print(response)

        data = json.loads(response.text)
        fff = data['data']['datas']

        ss = []
        for i in fff:
            # item['title'] = i.get('roomName')
            # item['count'] = i.get('totalCount')
            # item['image_url'] = i.get('screenshot')
            item = {
                'title': i.get('roomName'),
                'count': i.get('totalCount'),
                'image_url': i.get('screenshot')
            }
            ss.append(item)
        logging.info(ss)
        with open('data.json', 'w', encoding = 'utf-8') as f:
            f.write(json.dumps(ss, ensure_ascii = False))
            f.write('\r\n')


if __name__ == '__main__':
    # os.system('scrapy crawl xl')
    cmdline.execute('scrapy crawl xl'.split())
